﻿using MongoDB.Bson;
using MongoDB.Bson.Serialization;
using MongoDB.Bson.Serialization.Attributes;
using MongoDB.Driver;
using Newtonsoft.Json;
using Sustainalytics.Auth.Claims;
using Sustainalytics.BusinessResearchProductStates.Entities.ResearchState;
using Sustainalytics.ClientDW.DataAccessLayer;
using Sustainalytics.ClientDW.Entities;
using Sustainalytics.DataLayer;
using Sustainalytics.DataLayer.Shared;
using Sustainalytics.Entities;
using Sustainalytics.Entities.ScreeningTool;
using Sustainalytics.Portfolios.BL;
using Sustainalytics.Portfolios.BL.Model;
using Sustainalytics.ScreeningTool.BL.Model;
using Sustainalytics.Utils;
using System;
using System.Collections.Generic;
using System.Configuration;
using System.Linq;
using System.Security.Claims;
using System.Threading.Tasks;
using Sustainalytics.ESG.Entities;
using Sustainalytics.ClientDW.Entities.Universes;
using Sustainalytics.Entities.RiskRatings;
using Sustainalytics.Utils.Serilog;
using Sustainalytics.ScreeningTool.BL.Serilog;
using Sustainalytics.Entities.SustainableProducts;
using Sustainalytics.Entities.ProductInvolvement;
using System.Text.RegularExpressions;

namespace Sustainalytics.ScreeningTool.BL
{
    public class CompanyScreening : ICompanyScreening
    {
        #region Fields

        public bool IsRunningInPregen { get; set; }
        private ICompanyScreeningRepository _companyScreeningRepository;
        private IScreeningDataRepository _screeningDataRepository;
        private IClientDwRepository _clientDWRepository;
        private IPortfolioReadHandler _portfolioReadHandler;
        private IUniversesRepository _universeRepository;
        private IFilterCriteriaRepository _filterCriteriaRepository;
        private IRepositoryCollection<GovernanceIndicatorTemplate> _governanceIndicatorTemplateRepository;
        private IMongoCollection<SustainableProductsFramework> _sustainableProductsFramework;
        private IMongoCollection<ProductInvolvementFramework> _productInvolvementFramework;
        private IMongoCollection<CorporateData> _corporateDataCollection;


        #endregion Fields

        #region Constructors
        private CompanyScreening() { }

        public CompanyScreening(ICompanyScreeningRepository companyScreeningRepository,
            IClientDwRepository clientDWRepository,
            IPortfolioReadHandler portfolioDetailsRepository,
            IUniversesRepository universeRepository,
            IFilterCriteriaRepository filterCriteriaRepository,
            IRepositoryCollection<GovernanceIndicatorTemplate> governanceIndicatorTemplateRepository,
            IScreeningDataRepository screeningDataRepository,
            IMongoCollection<CorporateData> corporateDataCollection,
            IMongoCollection<SustainableProductsFramework> sustainableProductsFramework,
            IMongoCollection<ProductInvolvementFramework> productInvolvementFramework)
        {
            _companyScreeningRepository = companyScreeningRepository;
            _clientDWRepository = clientDWRepository;
            _portfolioReadHandler = portfolioDetailsRepository;
            _universeRepository = universeRepository;
            _filterCriteriaRepository = filterCriteriaRepository;
            _governanceIndicatorTemplateRepository = governanceIndicatorTemplateRepository;
            _screeningDataRepository = screeningDataRepository;
            _corporateDataCollection = corporateDataCollection;
            _sustainableProductsFramework = sustainableProductsFramework;
            _productInvolvementFramework = productInvolvementFramework;
        }

        public static ICompanyScreening GetInstance(
            ICompanyScreeningRepository companyScreeningRepository,
            IClientDwRepository clientDWRepository,
            IPortfolioReadHandler portfolioDetailsRepository,
            IUniversesRepository universeRepository,
            IFilterCriteriaRepository filterCriteriaRepository,
            IRepositoryCollection<GovernanceIndicatorTemplate> governanceIndicatorTemplateRepository,
            IScreeningDataRepository screeningDataRepository,
            IMongoCollection<CorporateData> corporateDataCollection,
            IMongoCollection<SustainableProductsFramework> sustainableProductsFramework,
            IMongoCollection<ProductInvolvementFramework> productInvolvementFramework,
            bool isInPregen = false)
        {
            return new CompanyScreening
            {
                _companyScreeningRepository = companyScreeningRepository,
                _clientDWRepository = clientDWRepository,
                _portfolioReadHandler = portfolioDetailsRepository,
                _universeRepository = universeRepository,
                _filterCriteriaRepository = filterCriteriaRepository,
                _governanceIndicatorTemplateRepository = governanceIndicatorTemplateRepository,
                _screeningDataRepository = screeningDataRepository,
                _corporateDataCollection = corporateDataCollection,
                _sustainableProductsFramework = sustainableProductsFramework,
                _productInvolvementFramework = productInvolvementFramework,
                IsRunningInPregen = isInPregen
            };
        }

        #endregion

        #region add criteria

        public List<CriteriaTypeFour> LoadCriteriaTypeFour(Guid piFilterCriteriaId)
        {
            return _companyScreeningRepository.LoadCriteriaTypeFour(piFilterCriteriaId);
        }

        #endregion

        #region list criteria

        public CriteriaLevel1 GetCriteriaLevelOne(Guid criteriaLevel1Id)
        {
            try
            {
                CommonEventSerilog.Log.CallerName();
                return _companyScreeningRepository.GetCriteriaLevelOne(criteriaLevel1Id);
            }
            catch (Exception ex)
            {
                CommonEventSerilog.Log.CallerNameError(ex);
                throw;
            }
        }

        public CriteriaLevel2 GetCriteriaLevelTwo(Guid criteriaLevel2Id)
        {
            try
            {
                CommonEventSerilog.Log.CallerName();
                return _companyScreeningRepository.GetCriteriaLevelTwo(criteriaLevel2Id);
            }
            catch (Exception ex)
            {
                CommonEventSerilog.Log.CallerNameError(ex);
                throw;
            }
        }

        public CriteriaLevel3 GetCriteriaLevelThree(Guid criteriaLevel3Id)
        {
            return _companyScreeningRepository.GetCriteriaLevelThree(criteriaLevel3Id);
        }

        public List<CriteriaLevel1> ListCriteriaLevelOne()
        {
            try
            {
                CommonEventSerilog.Log.CallerName();

                return _companyScreeningRepository.ListCriteriaLevelOne();
            }
            catch (Exception ex)
            {
                CommonEventSerilog.Log.CallerNameError(ex);
                throw;
            }
        }

        public List<CriteriaLevel2> ListCriteriaLevelTwo(Guid? criteriaLevel1Id)
        {
            try
            {
                CommonEventSerilog.Log.CallerName();
                return _companyScreeningRepository.ListCriteriaLevelTwo(criteriaLevel1Id);
            }
            catch (Exception ex)
            {
                CommonEventSerilog.Log.CallerNameError(ex);
                throw;
            }
        }

        public List<BaseCriteria> ListBaseCriteria(Guid? criteriaLevel2Id)
        {
            try
            {
                CommonEventSerilog.Log.CallerName();
                return _companyScreeningRepository.ListBaseCriteria(criteriaLevel2Id);
            }
            catch (Exception ex)
            {
                CommonEventSerilog.Log.CallerNameError(ex);
                throw;
            }
        }

        #endregion

        #region load criteria

        public T LoadTemplateCriteria<T>(Guid criteriaTemplateId) where T : IEntity
        {
            return _companyScreeningRepository.LoadTemplateCriteria<T>(criteriaTemplateId);
        }

        public List<CriteriaChartData> LoadCriteriaTypeOneData(Guid criteriaTemplateId, List<int> universeIds)
        {
            var criteriaTypeOne = LoadTemplateCriteria<CriteriaTypeOne>(criteriaTemplateId);
            return ComputeCriteriaTypeOneData(criteriaTypeOne, universeIds);
        }

        public CriteriaTypeThree LoadCriteriaTypeThreeSingleCollection(string collectionName)
        {
            return _companyScreeningRepository.LoadCriteriaTypeThreeSingleCollection(collectionName);
        }

        public List<CriteriaBasic> LoadCriteriaTypeThreeDataSingleItems(CriteriaTypeThree criteriaTypeThreeData)
        {
            return _clientDWRepository.LoadFilterTypeThreeSettings(criteriaTypeThreeData.CriteriaEntityField);
        }

        public List<string> LoadCriteriaTypeThreeData(Guid templateId, List<int> universeIds, int skip, int take, string filter, out int totalCount)
        {
            var criteriaTypeThree = LoadTemplateCriteria<CriteriaTypeThree>(templateId);
            return _clientDWRepository.LoadCriteriaTypeThreeData(criteriaTypeThree.CriteriaEntityField, null, universeIds, skip, take, filter, out totalCount);
        }


        public List<CriteriaChartData> LoadCriteriaTypeSixData(Guid criteriaTemplateId, List<int> universeIds)
        {
            var criteriaTypeSix = LoadTemplateCriteria<CriteriaTypeSix>(criteriaTemplateId);
            return ComputeCriteriaTypeSixData(criteriaTypeSix, universeIds);
        }

        public List<string> LoadCriteriaTypeSevenData(Guid templateId, List<int> universeIds, int skip, int take, string filter, out int totalCount)
        {
            var criteriaTypeSeven = LoadTemplateCriteria<CriteriaTypeSeven>(templateId);
            return _clientDWRepository.LoadCriteriaTypeSevenData(criteriaTypeSeven.CriteriaEntityField, criteriaTypeSeven.CollectionName, universeIds, skip, take, filter, out totalCount);
        }

        public CriteriaType16 LoadCriteriaType16(Guid criteriaTemplateId)
        {
            return _companyScreeningRepository.LoadCriteriaType16(criteriaTemplateId);
        }

        public List<CriteriaChartData> LoadCriteriaType16Data(Guid criteriaTemplateId, List<int> universeIds)
        {
            var criteriaType16 = LoadCriteriaType16(criteriaTemplateId);
            return ComputeCriteriaType16Data(criteriaType16, universeIds);
        }

        #endregion

        #region load template criteria

        public CriteriaTypeFive LoadTemplateCriteriaTypeFive(Guid criteriaTemplateId)
        {
            CompanyScreeningSerilog.Log.LoadTemplateCriteriaTypeFiveService(criteriaTemplateId);
            var criteria = _companyScreeningRepository.LoadTemplateCriteria<CriteriaTypeFive>(criteriaTemplateId);

            var templateIndicatorNumber = criteria.CriteriaEntityField.Split(' ').FirstOrDefault();

            var indicatorTemplate = _clientDWRepository.GetIndicatorTemplate(templateIndicatorNumber);
            if (indicatorTemplate != null)
                criteria.AnswerCategories = indicatorTemplate.AnswerCategories;

            return criteria;
        }

        public CriteriaType25 LoadTemplateCriteriaType25(Guid criteriaTemplateId)
        {
            CompanyScreeningSerilog.Log.LoadTemplateCriteriaType25Service(criteriaTemplateId);
            var criteria = _companyScreeningRepository.LoadTemplateCriteria<CriteriaType25>(criteriaTemplateId);

            var templateIndicatorNumber = criteria.CriteriaEntityField.Split(' ').FirstOrDefault();

            var indicatorTemplate = _clientDWRepository.GetIndicatorTemplate(templateIndicatorNumber);
            if (indicatorTemplate != null)
                criteria.AnswerCategories = indicatorTemplate.AnswerCategories;

            return criteria;
        }

        public CriteriaLevel2 GetCriteriaLevel2Id(Guid id)
        {
            return _companyScreeningRepository.GetCriteriaLevelTwo(id);
        }

        public CriteriaLevel2 GetCriteriaLevelTwoByLevel3Id(Guid criteriaLevel3Id)
        {
            return _companyScreeningRepository.GetCriteriaLevelTwoByLevel3Id(criteriaLevel3Id);
        }

        #endregion

        #region process criteria

        public List<Guid> ProcessCriteriaTypeOne(CriteriaTypeOne criteria, Guid? portfolioId, List<int> universeIds, List<Guid> criteriaCompanies)
        {
            var results = _screeningDataRepository.Find(universeIds, filter =>
                filter.Gte(criteria.CriteriaEntityField, criteria.MinValue) &
                filter.Lte(criteria.CriteriaEntityField, criteria.MaxValue));

            return ProcessCriteriaCommon(results, portfolioId, criteriaCompanies);
        }

        public List<Guid> ProcessCriteriaTypeTwo(CriteriaTypeTwo criteria, Guid? portfolioId, List<int> universeIds, List<Guid> criteriaCompanies)
        {
            var score = criteria.Score > 0 ? 5 - criteria.Score + 1 : criteria.Score;
            var results = _screeningDataRepository.Find(universeIds, filter =>
                score != null ? filter.Lt(criteria.CriteriaEntityField, score) : filter.Empty);

            return ProcessCriteriaCommon(results, portfolioId, criteriaCompanies);
        }

        public List<Guid> ProcessCriteriaTypeThree(CriteriaTypeThree criteria, Guid? portfolioId, List<int> universeIds, List<Guid> criteriaCompanies)
        {
            var results = _screeningDataRepository.Find(universeIds, filter =>
                filter.In(criteria.CriteriaEntityField, criteria.SelectedItems));

            return ProcessCriteriaCommon(results, portfolioId, criteriaCompanies);
        }

        public List<Guid> ProcessCriteriaTypeFour(CriteriaTypeFour criteria, Guid? portfolioId, List<int> universeIds, List<Guid> criteriaCompanies, ScreeningReportData screeningData)
        {
            List<Guid> results = new List<Guid>();
            var sustainableProductIndicator = _sustainableProductsFramework
              .Find(x => x.Categories.Any(y => y.Symbol == criteria.PiFilterCriteriaRange.CategorySymbol))
              .FirstOrDefault();

            if (sustainableProductIndicator != null)
                results = ProcessSustainableProductsIndicator(
                    criteria.CriteriaEntityField,
                    universeIds,
                    criteria.PiFilterCriteriaRange.Start,
                    screeningData?.CompanyData);

            var productInvolvementIndicator = _productInvolvementFramework
               .Find(x => x.Categories.Any(y => y.Symbol == criteria.PiFilterCriteriaRange.CategorySymbol))
               .FirstOrDefault();

            if (productInvolvementIndicator != null)
                results = ProcessProductInvolvementIndicator(
                    criteria.CriteriaEntityField,
                    universeIds,
                    criteria.PiFilterCriteriaRange.RevenueRange,
                    criteria.PiFilterCriteriaRange.Start,
                    screeningData?.CompanyData);

            return ProcessCriteriaCommon(results, portfolioId, criteriaCompanies);
        }

        private List<Guid> ProcessProductInvolvementIndicator(
            string criteriaEntityField,
            List<int> universeIds,
            bool isRevenueRange,
            double start,
            List<BsonDocument> companyData)
        {
            var regularExpressions = new Dictionary<int, string>()
            {
                { 0, "^(?s).*" },
                { 5, "^(5-|10-|25-|50-)?[0-9]?[0-9].[0-9]?%" },
                { 10, "^(10-|25-|50-)?[0-9]?[0-9].[0-9]?%" },
                { 25, "^(25-|50-)?[0-9]?[0-9].[0-9]?%" },
                { 50, "^(50-)?[0-9]?[0-9].[0-9]?%" }
            };

            IEnumerable<Guid> involvedCompanies = null;
            IEnumerable<Guid> companies = null;

            var companyIds = _corporateDataCollection
                .Find(cd => cd.ResearchParentId == null
                     && cd.UniverseIds.Any(x => universeIds.Contains(x))
                     && cd.ProductInvolvementResearchState.ResearchState == ResearchState.ResearchAvailable
                     && cd.ProductInvolvementResearchState.ClientFacingProducts.Contains(ClientFacingProduct.GA))
                .Project(cd => cd.Id)
                .ToList();

            if(!isRevenueRange)
                involvedCompanies = _screeningDataRepository.Find(universeIds, filter => filter.And(
                    Builders<ScreeningData>.Filter.Exists(criteriaEntityField),
                    Builders<ScreeningData>.Filter.In(x => x.Id, companyIds)));

            if (isRevenueRange)
            {
                var regex =  new BsonRegularExpression(regularExpressions[(int)start], "i");

                involvedCompanies = _screeningDataRepository.Find(universeIds, filter => filter.And(
                    Builders<ScreeningData>.Filter.Exists(criteriaEntityField),
                    Builders<ScreeningData>.Filter.Regex(criteriaEntityField, regex),
                    Builders<ScreeningData>.Filter.In(x => x.Id, companyIds)));
            }

            var coverageCompanies = GetCoverageCompanies(involvedCompanies);
            involvedCompanies = involvedCompanies.Union(coverageCompanies);

            if (companyData == null)
                companies = _corporateDataCollection.Find(Builders<CorporateData>.Filter.And(
                            Builders<CorporateData>.Filter.Eq(x => x.ProductInvolvementResearchState.ResearchState, ResearchState.ResearchAvailable),
                            Builders<CorporateData>.Filter.AnyIn(x => x.UniverseIds, universeIds)))
                        .Project(x => x.Id).ToList();
            else
            {
                companies = companyData.Where(x =>
                    (x["ResearchStates"].AsBsonDocument["ProductInvolvement"].AsString == "ResearchAvailable" ||
                    (x.Contains("ParentResearchStates") &&
                    x["ParentResearchStates"].AsBsonDocument["ProductInvolvement"].AsString == "ResearchAvailable")) &&
                    x["UniverseIds"].AsBsonArray.Any(y => universeIds.Contains(y.AsInt32))
                ).Select(x => x["_id"].AsGuid).ToList();
            }

            var result = companies.Except(involvedCompanies).ToList();
            var coverage = GetCoverageCompanies(result);
            result.AddRange(coverage);

            return result.Distinct().ToList();
        }

        private IEnumerable<Guid> GetCoverageCompanies(IEnumerable<Guid> involvedCompanies)
        {

            var query = Builders<CorporateData>.Filter.Ne(a => a.ResearchParentId, null);
            return _corporateDataCollection
                .Find(query)
                .Project<CorporateData>(Builders<CorporateData>.Projection.Include(x => x.Id).Include(x => x.ResearchParentId))
                .ToList()
                .Where(a => involvedCompanies.Contains(a.ResearchParentId.GetValueOrDefault()) &&
                    a.ResearchParentId.GetValueOrDefault() != Guid.Empty)
                .Select(a => a.Id);
        }

        private List<Guid> ProcessSustainableProductsIndicator(
            string criteriaEntityField,
            List<int> universeIds,
            double start,
            List<BsonDocument> companyData)
        {
            var regularExpressions = new Dictionary<int, string>()
            {
                { 0, "^()?[0-9]?[0-9].[0-9]?%" },
                { 5, "^(0-)?[0-9]?[0-9].[0-9]?%" },
                { 10, "^(0-|5-)?[0-9]?[0-9].[0-9]?%" },
                { 25, "^(0-|5-|10-)?[0-9]?[0-9].[0-9]?%" },
                { 50, "^(0-|5-|10-|25-)?[0-9]?[0-9].[0-9]?%" }
            };

            IEnumerable<Guid> involvedCompanies = null;
            IEnumerable<Guid> companies = null;

            var companyIds = _corporateDataCollection
                .Find(cd => cd.ResearchParentId == null
                     && cd.UniverseIds.Any(x => universeIds.Contains(x))
                     && cd.SolutionsResearchState.ResearchState == ResearchState.ResearchAvailable
                     && cd.SolutionsResearchState.ClientFacingProducts.Contains(ClientFacingProduct.GA))
                .Project(cd => cd.Id)
                .ToList();

            var regex = new BsonRegularExpression(regularExpressions[(int)start], "i");

            involvedCompanies = _screeningDataRepository.Find(universeIds, filter => filter.And(
                Builders<ScreeningData>.Filter.Or(
                    Builders<ScreeningData>.Filter.Not(Builders<ScreeningData>.Filter.Exists(criteriaEntityField)),
                    Builders<ScreeningData>.Filter.Regex(criteriaEntityField, regex)
                ),
                Builders<ScreeningData>.Filter.In(x => x.Id, companyIds)));

            var coverageCompanies = GetCoverageCompanies(involvedCompanies);
            involvedCompanies = involvedCompanies.Union(coverageCompanies);

            if (companyData == null)
                companies = _corporateDataCollection.Find(Builders<CorporateData>.Filter.And(
                            Builders<CorporateData>.Filter.Eq(x => x.SolutionsResearchState.ResearchState, ResearchState.ResearchAvailable),
                            Builders<CorporateData>.Filter.AnyIn(x => x.UniverseIds, universeIds)))
                        .Project(x => x.Id).ToList();
            else
            {
                companies = companyData.Where(x =>
                    (x["ResearchStates"].AsBsonDocument["Solutions"].AsString == "ResearchAvailable" ||
                    (x.Contains("ParentResearchStates") &&
                    x["ParentResearchStates"].AsBsonDocument["Solutions"].AsString == "ResearchAvailable")) &&
                    x["UniverseIds"].AsBsonArray.Any(y => universeIds.Contains(y.AsInt32))
                ).Select(x => x["_id"].AsGuid).ToList();
            }

            var result = companies.Except(involvedCompanies).ToList();
            var coverage = GetCoverageCompanies(result);
            result.AddRange(coverage);

            return result.Distinct().ToList();

        }

        public List<Guid> ProcessCriteriaTypeSix(CriteriaTypeSix criteria, Guid? portfolioId, List<int> universeIds, List<Guid> criteriaCompanies)
        {
            string criteriaEntityField = criteria.CriteriaEntityField;
            if (criteria.CriteriaEntityField.EndsWith("Rank", StringComparison.Ordinal) || 
                criteriaEntityField.EndsWith("Rank-Universe", StringComparison.Ordinal) || 
                criteriaEntityField.EndsWith("Rank-Subindustry", StringComparison.Ordinal))
                criteriaEntityField += ".Rank";

            double minValue = criteria.MinValue, maxValue = criteria.MaxValue;
            var results = _screeningDataRepository.Find(universeIds, filter =>
                filter.Gte(criteriaEntityField, minValue) &
                filter.Lte(criteriaEntityField, maxValue));
            
            return ProcessCriteriaCommon(results, portfolioId, criteriaCompanies);
        }

        public List<Guid> ProcessCriteriaTypeSeven(CriteriaTypeSeven criteria, Guid? portfolioId, List<int> universeIds, List<Guid> criteriaCompanies)
        {
            var selectedItems = criteria.SelectedItems.Select<string, BsonValue>(x =>
            {
                int intVal;
                if (int.TryParse(x, out intVal))
                    return intVal;

                return x;
            });

            var results = _screeningDataRepository.Find(universeIds, filter =>
                !filter.In(criteria.CriteriaEntityField, selectedItems));

            return ProcessCriteriaCommon(results, portfolioId, criteriaCompanies);
        }

        public List<Guid> ProcessCriteriaTypeEight(CriteriaTypeEight criteria, Guid? portfolioId, List<int> universeIds, List<Guid> criteriaCompanies)
        {
            var score = criteria.Score > 0 ? 5 - criteria.Score + 1 : criteria.Score;
            var results = _screeningDataRepository.Find(universeIds, filter =>
                score != null ? filter.Lt(criteria.CriteriaEntityField, score) : filter.Empty);

            return ProcessCriteriaCommon(results, portfolioId, criteriaCompanies);
        }

        public List<Guid> ProcessCriteriaTypeNine(CriteriaTypeNine criteria, Guid? portfolioId, List<int> universeIds)
        {
            var builder = Builders<ScreeningData>.Filter;
            var exists = builder.Exists(criteria.CriteriaEntityField);
            var lowerThen = criteria.Score != null
                ? builder.Lt(criteria.CriteriaEntityField, criteria.Score.Value)
                : builder.Empty;
            var hasControversy = builder.Eq(x => x.ResearchStates.Controversy, ResearchState.ResearchAvailable);

            var results = _screeningDataRepository.Find(universeIds,
                filter => (exists & lowerThen) | (!exists & hasControversy));

            return ProcessCriteriaCommon(results, portfolioId);
        }

        public List<Guid> ProcessCriteriaType16(CriteriaType16 criteria, Guid? portfolioId, List<int> universeIds, List<Guid> criteriaCompanies)
        {
            string criteriaEntityField = criteria.CriteriaEntityField;

            if (criteria.CriteriaEntityField.StartsWith("CG - Rank - ", StringComparison.Ordinal))
                criteriaEntityField += ".Rank";

            var results = _screeningDataRepository.Find(universeIds, filter =>
                filter.Gte(criteriaEntityField, criteria.MinValue) &
                filter.Lte(criteriaEntityField, criteria.MaxValue));

            return ProcessCriteriaCommon(results, portfolioId, criteriaCompanies);
        }

        public List<Guid> ProcessCriteriaType17(CriteriaType17 criteria, Guid? portfolioId, List<int> universeIds)
        {
            var results = _screeningDataRepository.Find(universeIds, filter =>
                criteria.Score != null ? filter.Gt(criteria.CriteriaEntityField, criteria.Score.Value) : filter.Empty);

            return ProcessCriteriaCommon(results, portfolioId);
        }

        private List<Guid> ProcessCriteriaCommon(IEnumerable<Guid> results, Guid? portfolioId, List<Guid> criteriaCompanies = null)
        {
            var coverage = _clientDWRepository.GetCoverageCompanies(results);
            results = results.Concat(coverage).Distinct();

            if (portfolioId != null && portfolioId != Guid.Empty)
            {
                var portfolioCompaniesIds = _portfolioReadHandler.GetCompanyIdsByPortfolio(portfolioId.Value);
                results = results.Where(e => portfolioCompaniesIds.Any(c => c.Equals(e))).ToList();
            }

            if (criteriaCompanies != null && criteriaCompanies.Any())
            {
                results = results.Where(e => criteriaCompanies.Any(c => c.Equals(e))).ToList();
            }

            return results.ToList();
        }

        #endregion

        #region run

        public List<PortfolioSecurityModel> GetClientPortfolioSecurities(Guid portfolioId, PortfolioUserClaims userClaims)
        {
            return _portfolioReadHandler.GetPortfolioSecurities(portfolioId, userClaims).ToList();
        }

        private BsonArray _bsonResultOutput;

        protected List<PortfolioDetails> GetPortfolioDataFromFilter(Filter filterDetails, List<Universe> accountUniverseIds)
        {
            List<PortfolioDetails> portfolioData = null;

            var portfolioId = filterDetails.PortfolioId.GetValueOrDefault();
            var universes = accountUniverseIds.Select(a => a.Id).ToList();
            if (portfolioId != Guid.Empty && universes.Any())
            {
                var securities = _portfolioReadHandler.GetPortfolioSecurities(portfolioId, universes);
                portfolioData = securities.Select(x => x.GetPortfolioDetails()).ToList();
            }

            return portfolioData;
        }

        private IEnumerable<FilterCriteria> AddCriteriaType4ExtraField(FilterCriteria criteria)
        {
            var elements = criteria.CriteriaEntityField.Split(' ');
            if (elements.Count() < 2)
                yield return null;

            yield return new FilterCriteria()
            {
                Id = Guid.NewGuid(),
                CriteriaEntity = criteria.CriteriaEntity,
                CriteriaLevel2Id = criteria.CriteriaLevel2Id,
                CriteriaLevel3Id = criteria.CriteriaLevel3Id,
                CriteriaType = criteria.CriteriaType,
                FilterGroupId = criteria.FilterGroupId,
                FilterId = criteria.FilterId,
                HasFullRangeChart = criteria.HasFullRangeChart,
                IsDataPoint = criteria.IsDataPoint,
                Name = criteria.Name,
                Order = criteria.Order,
                ResearchProducts = criteria.ResearchProducts,
                SelectedValue = criteria.SelectedValue,
                SelectedValueInformation = criteria.SelectedValueInformation,
                TemplateCriteriaId = criteria.TemplateCriteriaId,
                CriteriaEntityField = $"{elements[0]} {elements[1]} Category of Involvement",
                IsTemporary = true
            };
        }

        public List<BsonValue> RunSelectedCriteriaOnScreeningData(
            Guid filterId,
            Guid portfolioId, 
            bool report, 
            bool memoryResultOutput, 
            Guid accountId,
            ScreeningReportData screeningReportData)
        {
            if (screeningReportData == null)
                screeningReportData = new ScreeningReportData();

            var currentClaims = ClaimsPrincipal.Current.Claims;
            if (accountId == Guid.Empty)
            {
                var stringAccountId = currentClaims.Where(s => s.Type == CustomClaimTypes.AccountId)
                                           .Select(s => s.Value)
                                           .SingleOrDefault();
                Guid.TryParse(stringAccountId, out accountId);
            }

            var filterDetails = _companyScreeningRepository.GetFilterByFilterId(filterId);
            if (filterDetails == null)
                throw new KeyNotFoundException($"The filter with id {filterId} cannot be found");

            var filterCriterias = _filterCriteriaRepository.GetAllFilterCriteriaByFilterId(filterId);

            var criteriaType4 = filterCriterias.Where(x => x.CriteriaType == CriteriaTypeEnum.CriteriaType4).ToList();

            if (criteriaType4.Any())
            {
                criteriaType4.ForEach(x =>
                {
                    filterCriterias.AddRange(AddCriteriaType4ExtraField(x));
                });
            }

            var accountUniverseIds = _universeRepository.GetUniversesByAccountId(accountId);
            var portfolioData = GetPortfolioDataFromFilter(filterDetails, accountUniverseIds);
            List<int> universeIds = currentClaims.GetUniverseIds().ToList();
            var accountName = currentClaims.Where(s => s.Type == CustomClaimTypes.AccountName).Select(s => s.Value).SingleOrDefault();
            

            List<Guid> companiesById = null;
            if (filterDetails.PortfolioId.HasValue)
                companiesById = _portfolioReadHandler.GetCompanyIdsByPortfolio(filterDetails.PortfolioId.Value).ToList();

            if (accountName != null)
                screeningReportData.ClientName = accountName;
            screeningReportData.PortfolioName = filterDetails.PortfolioName;
            screeningReportData.FilterName = filterDetails.Name;
            screeningReportData.FilterSettings = filterDetails;

            if (screeningReportData.CompanyData == null || !screeningReportData.CompanyData.Any())
            {
                var helper = new CUSIPISINClaimsCheckerBL();
                helper.checkIsinCusipClaims(ClaimsPrincipal.Current);

                screeningReportData.CompanyData = _screeningDataRepository.GetCompaniesData(filterDetails, universeIds, accountUniverseIds, portfolioData, filterCriterias, companiesById, helper.HasISINClaim, helper.HasCUSIPClaim);
            }
            if (report)
                screeningReportData.CompanyData = screeningReportData.CompanyData.Take(10).ToList();

            // Get universes for each criteria
            var criteriasToFeatures = _companyScreeningRepository
                .ListFeatureUniversesByCriteriaLevelTwoId(filterCriterias.Select(p => p.CriteriaLevel2Id).ToList())
                .ToDictionary(c => c.Id, c => c.UniverseFeature);


            var companiesInResultList = new List<CompaniesInResultInformation>();
            // Process companies for each Filter Criteria
            if (filterCriterias.Any())
                ProcessCompaniesByCriteriaType(
                    portfolioId, 
                    filterCriterias, 
                    criteriasToFeatures, 
                    companiesInResultList, 
                    filterDetails, 
                    accountUniverseIds.Select(x=> x.Id).ToList(), 
                    screeningReportData);

            var criteriasToUniverses = new Dictionary<Guid, List<int>>();

            if (filterCriterias.Any())
                criteriasToUniverses = criteriasToFeatures.ToDictionary(
                    criterias => criterias.Key,
                    criterias => _universeRepository.GetUniverseIdsByFeatureIds(criterias.Value, accountId != Guid.Empty ? accountId : (Guid?)null));

            _bsonResultOutput = new BsonArray();
            try
            {
                Parallel.ForEach(screeningReportData.CompanyData, corporateDataKvp =>
                {
                    var corporateData = BsonSerializer.Deserialize<ScreeningData>(corporateDataKvp);

                    if (string.IsNullOrEmpty(corporateData.CompanyName) ||
                        string.IsNullOrEmpty(corporateData.CompanyNameLowerCase) ||
                        string.IsNullOrEmpty(corporateData.CompanyNameUrlFriendly))
                        return;

                    var criteriaResultOutputList = new List<CriteriaResultOutput>();

                    Parallel.ForEach(companiesInResultList, companiesInResult =>
                    {
                        if (companiesInResult == null)
                            return;

                        var criteriaId = companiesInResult.CriteriaId;
                        var criteriaCompanies = companiesInResult.CriteriaCompanies;
                        var filterCriteriaGroupId = companiesInResult.FilterCriteriaGroupId;
                        var isDataPoint = companiesInResult.IsDataPoint;
                        var noDataCompanies = companiesInResult.NoDataCompanies;

                        var criteriaResultOutput = new CriteriaResultOutput
                        {
                            CompaniesCount = !isDataPoint ? criteriaCompanies.Count.ToString() : "Data Point",
                            SelectedValueInformation = companiesInResult.FilterSelectedValueInformation,
                            Order = companiesInResult.FilterCriteriaOrder,
                            CriteriaType = companiesInResult.CriteriaType,
                            CriteriaEntityField = companiesInResult.CriteriaEntityField,
                            FilterCriteriaId = companiesInResult.FilterCriteriaId,
                            ResearchProducts = companiesInResult.ResearchProducts,
                            IsTemporary = companiesInResult.IsTemporary
                        };

                        if (criteriaCompanies.Contains(corporateData.Id))
                        {
                            criteriaResultOutput.IsValid = criteriaCompanies.Contains(corporateData.Id)
                                ? ResultIsValid.Valid
                                : ResultIsValid.NotValid;
                        }
                        if (noDataCompanies != null && noDataCompanies.Contains(corporateData.Id))
                        {
                            criteriaResultOutput.IsValid = ResultIsValid.NotAvailable;
                        }

                        var researchState = companiesInResult.ResearchProducts.GetResearchState(corporateData.ToBsonDocument());
                        if (researchState != ResearchState.ResearchAvailable)
                        {
                            criteriaResultOutput.IsValid = ResultIsValid.NotAvailable;
                        }

                        if (!criteriasToUniverses[criteriaId].Intersect(corporateData.UniverseIds).Any())
                        {
                            criteriaResultOutput.IsValid = ResultIsValid.NotAvailable;
                            criteriaResultOutput.StringValue = "No Access";
                        }

                        if (isDataPoint)
                            criteriaResultOutput.IsValid = ResultIsValid.NotAvailable;

                        if (string.IsNullOrWhiteSpace(criteriaResultOutput.Name))
                            criteriaResultOutput.Name = companiesInResult.CriteriaName;

                        if (filterCriteriaGroupId != null)
                            criteriaResultOutput.FilterCriteriaGroupId = filterCriteriaGroupId.Value;

                        lock (criteriaResultOutputList)
                            criteriaResultOutputList.Add(criteriaResultOutput);
                    });

                    ResultIsValid isValid = ResultIsValid.NotValid;
                    if (criteriaResultOutputList.Where(x => !x.IsTemporary).All(c => c.IsValid == ResultIsValid.Valid || c.IsValid == ResultIsValid.NotAvailable))
                        isValid = ResultIsValid.Valid;

                    if (criteriaResultOutputList.All(c => c.IsValid == ResultIsValid.NotAvailable))
                        isValid = ResultIsValid.NotAvailable;

                    var document = new BsonDocument
                    {
                        { "_id",                    Guid.NewGuid()                                        },
                        { "IsValid",                isValid                                               },
                        { "Criteria",               JsonConvert.SerializeObject(criteriaResultOutputList) },
                        { "FilterId",               filterId                                              },
                        { "CompanyId",              corporateData.Id                                      },
                        { "CapitalIqId",            corporateData.CompanyId                               },
                        { "CompanyName",            corporateData.CompanyName                             },
                        { "CompanyNameLowerCase",   corporateData.CompanyNameLowerCase                    },
                        { "CompanyNameUrlFriendly", corporateData.CompanyNameUrlFriendly                  }
                    };
                    lock (_bsonResultOutput)
                        _bsonResultOutput.Add(document);
                });

                if (!_bsonResultOutput.Any())
                {
                    var document = new BsonDocument
                {
                    { "_id", Guid.Empty },
                    { "IsValid", ResultIsValid.NotAvailable },
                    { "Criteria",JsonConvert.SerializeObject(
                        companiesInResultList.Select(a=>new CriteriaResultOutput()
                        {
                            CompaniesCount =a.IsDataPoint?"Data Point":a.CriteriaCompanies.Count.ToString(),
                            SelectedValueInformation = a.FilterSelectedValueInformation,
                            Order = a.FilterCriteriaOrder,
                            CriteriaType = a.CriteriaType,
                            CriteriaEntityField = a.CriteriaEntityField,
                            FilterCriteriaId = a.FilterCriteriaId,
                            Name = a.CriteriaName,
                            ResearchProducts = a.ResearchProducts,
                            FilterCriteriaGroupId = a.FilterCriteriaGroupId.GetValueOrDefault(),
                            IsTemporary = a.IsTemporary
                        }))
                    },
                    { "FilterId", filterId},
                    { "CompanyId", Guid.Empty},
                    { "CapitalIqId", 0},
                    { "CompanyName", string.Empty},
                    { "CompanyNameLowerCase", string.Empty},
                    { "CompanyNameUrlFriendly", string.Empty}
                };
                    _bsonResultOutput.Add(document);
                }

                if (!memoryResultOutput)
                    _companyScreeningRepository.SaveResultOutput(filterId, _bsonResultOutput);

                lock (_bsonResultOutput)
                {
                    return _bsonResultOutput.ToList();
                }
            }
            finally
            {
                _bsonResultOutput = null;
                GC.Collect();
                GC.WaitForPendingFinalizers();
                GC.Collect();
            }
        }
        
        private readonly Dictionary<Guid, List<Guid>> _typeFiveNoDataCompanies = new Dictionary<Guid, List<Guid>>();
        private readonly Dictionary<Guid, List<Guid>> _type25NoDataCompanies = new Dictionary<Guid, List<Guid>>();
        
        private readonly Dictionary<Guid, List<Guid>> _dataCompanies = new Dictionary<Guid, List<Guid>>();

        private void ProcessCompaniesByCriteriaType(Guid portfolioId, List<FilterCriteria> currentFilterCriterias,
            Dictionary<Guid, int> criteriasToFeatures, List<CompaniesInResultInformation> companiesInResult, Filter filter, List<int> universesToUseInPregen = null, ScreeningReportData screeningReportData = null)
        {
            var typeFiveCompaniesForCriterias = new Dictionary<Guid, List<Guid>>();
            var typeFourNoDataCompanies = new Dictionary<Guid, List<Guid>>();
            var typeSixNoDataCompanies = new Dictionary<Guid, List<Guid>>();
            var typeEightNoDataCompanies = new Dictionary<Guid, List<Guid>>();
            var typeTwoNoDataCompanies = new Dictionary<Guid, List<Guid>>();
            var typeThreeNoDataCompanies = new Dictionary<Guid, List<Guid>>();
            var typeOneNoDataCompanies = new Dictionary<Guid, List<Guid>>();
            var typeSevenNoDataCompanies = new Dictionary<Guid, List<Guid>>();
            var typeNineNoDataCompanies = new Dictionary<Guid, List<Guid>>();
            var type16NoDataCompanies = new Dictionary<Guid, List<Guid>>();
            var type17NoDataCompanies = new Dictionary<Guid, List<Guid>>();            
            var type25CompaniesForCriterias = new Dictionary<Guid, List<Guid>>();

            if (currentFilterCriterias.Any(p => p.CriteriaType == CriteriaTypeEnum.CriteriaType1))
                typeOneNoDataCompanies = NewProcessCriteria(currentFilterCriterias, CriteriaTypeEnum.CriteriaType1, filter, screeningReportData);

            if (currentFilterCriterias.Any(p => p.CriteriaType == CriteriaTypeEnum.CriteriaType2))
                typeTwoNoDataCompanies = NewProcessCriteria(currentFilterCriterias, CriteriaTypeEnum.CriteriaType2, filter, screeningReportData);

            if (currentFilterCriterias.Any(p => p.CriteriaType == CriteriaTypeEnum.CriteriaType3))
                typeThreeNoDataCompanies = NewProcessCriteria(currentFilterCriterias, CriteriaTypeEnum.CriteriaType3, filter, screeningReportData);

            if (currentFilterCriterias.Any(p => p.CriteriaType == CriteriaTypeEnum.CriteriaType4))
                typeFourNoDataCompanies = NewProcessCriteria(currentFilterCriterias, CriteriaTypeEnum.CriteriaType4, filter, screeningReportData);

            if (currentFilterCriterias.Any(p => p.CriteriaType == CriteriaTypeEnum.CriteriaType5))
                typeFiveCompaniesForCriterias = NewProcessCriteria(currentFilterCriterias, CriteriaTypeEnum.CriteriaType5, filter, screeningReportData);

            if (currentFilterCriterias.Any(p => p.CriteriaType == CriteriaTypeEnum.CriteriaType25))
                type25CompaniesForCriterias = NewProcessCriteria(currentFilterCriterias, CriteriaTypeEnum.CriteriaType25, filter, screeningReportData);

            if (currentFilterCriterias.Any(p => p.CriteriaType == CriteriaTypeEnum.CriteriaType6))
                typeSixNoDataCompanies = NewProcessCriteria(currentFilterCriterias, CriteriaTypeEnum.CriteriaType6, filter, screeningReportData);

            if (currentFilterCriterias.Any(p => p.CriteriaType == CriteriaTypeEnum.CriteriaType7))
                typeSevenNoDataCompanies = NewProcessCriteria(currentFilterCriterias, CriteriaTypeEnum.CriteriaType7, filter, screeningReportData);

            if (currentFilterCriterias.Any(p => p.CriteriaType == CriteriaTypeEnum.CriteriaType8))
                typeEightNoDataCompanies = NewProcessCriteria(currentFilterCriterias, CriteriaTypeEnum.CriteriaType8, filter, screeningReportData);

            if (currentFilterCriterias.Any(p => p.CriteriaType == CriteriaTypeEnum.CriteriaType9))
                typeNineNoDataCompanies = NewProcessCriteria(currentFilterCriterias, CriteriaTypeEnum.CriteriaType9, filter, screeningReportData);

            if (currentFilterCriterias.Any(p => p.CriteriaType == CriteriaTypeEnum.CriteriaType16))
                type16NoDataCompanies = NewProcessCriteria(currentFilterCriterias, CriteriaTypeEnum.CriteriaType16, filter, screeningReportData);

            if (currentFilterCriterias.Any(p => p.CriteriaType == CriteriaTypeEnum.CriteriaType17))
                type17NoDataCompanies = NewProcessCriteria(currentFilterCriterias, CriteriaTypeEnum.CriteriaType17, filter, screeningReportData);

            Parallel.ForEach(currentFilterCriterias, currentFilterCriteria =>
            {
                var univs = ClaimsPrincipal.Current.Claims.GetUniverseIds(criteriasToFeatures[currentFilterCriteria.CriteriaLevel2Id]);

                if (IsRunningInPregen)
                    univs = universesToUseInPregen;
                if (!univs.Any() && (universesToUseInPregen != null && universesToUseInPregen.Any()))
                {
                    univs = universesToUseInPregen;
                }


                List<Guid> companies, noDataCompanies;

                if (!currentFilterCriteria.IsDataPoint)
                {
                    switch (currentFilterCriteria.CriteriaType)
                    {
                        case CriteriaTypeEnum.CriteriaType1:
                            var typeOne = new CriteriaTypeOne();
                            var criteriaTypeOne = JsonConvert.DeserializeAnonymousType(currentFilterCriteria.SelectedValue, typeOne);
                            criteriaTypeOne.CriteriaEntityField = currentFilterCriteria.CriteriaEntityField;

                            companies = ProcessCriteriaTypeOne(criteriaTypeOne, portfolioId, univs, _dataCompanies.ContainsKey(currentFilterCriteria.Id) ? _dataCompanies[currentFilterCriteria.Id] : new List<Guid>());
                            noDataCompanies = typeOneNoDataCompanies.ContainsKey(currentFilterCriteria.Id) ? typeOneNoDataCompanies[currentFilterCriteria.Id] : new List<Guid>();
                            AddCompaniesInResult(companiesInResult, currentFilterCriteria, companies, noDataCompanies);
                            break;

                        case CriteriaTypeEnum.CriteriaType2:
                            var typeTwo = new CriteriaTypeTwo();
                            var criteriaTypeTwo = JsonConvert.DeserializeAnonymousType(currentFilterCriteria.SelectedValue, typeTwo);
                            criteriaTypeTwo.CriteriaEntityField = currentFilterCriteria.CriteriaEntityField;
                            criteriaTypeTwo.EventIndicatorName = currentFilterCriteria.Name.Replace("-Answer Category", "");

                            companies = ProcessCriteriaTypeTwo(criteriaTypeTwo, portfolioId, univs, _dataCompanies.ContainsKey(currentFilterCriteria.Id) ? _dataCompanies[currentFilterCriteria.Id] : new List<Guid>());
                            noDataCompanies = typeTwoNoDataCompanies.ContainsKey(currentFilterCriteria.Id) ? typeTwoNoDataCompanies[currentFilterCriteria.Id] : new List<Guid>();
                            AddCompaniesInResult(companiesInResult, currentFilterCriteria, companies, noDataCompanies);
                            break;

                        case CriteriaTypeEnum.CriteriaType3:
                            var typeThree = new CriteriaTypeThree();
                            var criteriaTypeThree = JsonConvert.DeserializeAnonymousType(currentFilterCriteria.SelectedValue, typeThree);
                            criteriaTypeThree.CriteriaEntityField = currentFilterCriteria.CriteriaEntityField;

                            companies = ProcessCriteriaTypeThree(criteriaTypeThree, portfolioId, univs, _dataCompanies.ContainsKey(currentFilterCriteria.Id) ? _dataCompanies[currentFilterCriteria.Id] : new List<Guid>());
                            noDataCompanies = typeThreeNoDataCompanies.ContainsKey(currentFilterCriteria.Id) ? typeThreeNoDataCompanies[currentFilterCriteria.Id] : new List<Guid>();
                            AddCompaniesInResult(companiesInResult, currentFilterCriteria, companies, noDataCompanies);
                            break;

                        case CriteriaTypeEnum.CriteriaType4:
                            var typeFour = new CriteriaTypeFour();
                            var criteriaTypeFour = JsonConvert.DeserializeAnonymousType(currentFilterCriteria.SelectedValue, typeFour);
                            criteriaTypeFour.CriteriaEntityField = currentFilterCriteria.CriteriaEntityField;

                            companies = ProcessCriteriaTypeFour(criteriaTypeFour, portfolioId, univs, _dataCompanies.ContainsKey(currentFilterCriteria.Id) ? _dataCompanies[currentFilterCriteria.Id] : new List<Guid>(), screeningReportData);

                            lock (typeFourNoDataCompanies)
                            {
                                if (typeFourNoDataCompanies.ContainsKey(currentFilterCriteria.Id))
                                {
                                    typeFourNoDataCompanies[currentFilterCriteria.Id] =
                                        typeFourNoDataCompanies[currentFilterCriteria.Id].Except(companies).ToList();
                                }
                            }

                            noDataCompanies = typeFourNoDataCompanies.ContainsKey(currentFilterCriteria.Id) ? typeFourNoDataCompanies[currentFilterCriteria.Id] : new List<Guid>();
                            AddCompaniesInResult(companiesInResult, currentFilterCriteria, companies, noDataCompanies);
                            break;

                        case CriteriaTypeEnum.CriteriaType5:
                            var typeFive = new CriteriaTypeFive();
                            var criteriaTypeFive = JsonConvert.DeserializeAnonymousType(currentFilterCriteria.SelectedValue, typeFive);
                            criteriaTypeFive.CriteriaEntityField = currentFilterCriteria.CriteriaEntityField;
                            criteriaTypeFive.TemplateIndicatorNumber = currentFilterCriteria.Name.Split(' ')[0];

                            companies = typeFiveCompaniesForCriterias.ContainsKey(currentFilterCriteria.Id) ? typeFiveCompaniesForCriterias[currentFilterCriteria.Id] : new List<Guid>();
                            noDataCompanies = _typeFiveNoDataCompanies.ContainsKey(currentFilterCriteria.Id) ? _typeFiveNoDataCompanies[currentFilterCriteria.Id] : new List<Guid>();
                            AddCompaniesInResult(companiesInResult, currentFilterCriteria, companies, noDataCompanies);
                            break;

                        case CriteriaTypeEnum.CriteriaType25:
                            var type25 = new CriteriaType25();
                            var criteriaType25 = JsonConvert.DeserializeAnonymousType(currentFilterCriteria.SelectedValue, type25);
                            criteriaType25.CriteriaEntityField = currentFilterCriteria.CriteriaEntityField;
                            //criteriaType25.TemplateIndicatorNumber = currentFilterCriteria.Name.Split(' ')[0];

                            companies = type25CompaniesForCriterias.ContainsKey(currentFilterCriteria.Id) ? type25CompaniesForCriterias[currentFilterCriteria.Id] : new List<Guid>();
                            noDataCompanies = _type25NoDataCompanies.ContainsKey(currentFilterCriteria.Id) ? _type25NoDataCompanies[currentFilterCriteria.Id] : new List<Guid>();
                            AddCompaniesInResult(companiesInResult, currentFilterCriteria, companies, noDataCompanies);
                            break;

                        case CriteriaTypeEnum.CriteriaType6:
                            var typeSix = new CriteriaTypeSix();
                            var criteriaTypeSix = JsonConvert.DeserializeAnonymousType(currentFilterCriteria.SelectedValue, typeSix);
                            criteriaTypeSix.CriteriaEntityField = currentFilterCriteria.CriteriaEntityField;

                            companies = ProcessCriteriaTypeSix(criteriaTypeSix, portfolioId, univs, _dataCompanies.ContainsKey(currentFilterCriteria.Id) ? _dataCompanies[currentFilterCriteria.Id] : new List<Guid>());
                            noDataCompanies = typeSixNoDataCompanies.ContainsKey(currentFilterCriteria.Id) ? typeSixNoDataCompanies[currentFilterCriteria.Id] : new List<Guid>();
                            AddCompaniesInResult(companiesInResult, currentFilterCriteria, companies, noDataCompanies);
                            break;

                        case CriteriaTypeEnum.CriteriaType7:
                            var typeSeven = new CriteriaTypeSeven();
                            var criteriaTypeSeven = JsonConvert.DeserializeAnonymousType(currentFilterCriteria.SelectedValue, typeSeven);
                            criteriaTypeSeven.CriteriaEntityField = currentFilterCriteria.CriteriaEntityField;

                            companies = ProcessCriteriaTypeSeven(criteriaTypeSeven, portfolioId, univs, _dataCompanies.ContainsKey(currentFilterCriteria.Id) ? _dataCompanies[currentFilterCriteria.Id] : new List<Guid>());
                            noDataCompanies = typeSevenNoDataCompanies.ContainsKey(currentFilterCriteria.Id) ? typeSevenNoDataCompanies[currentFilterCriteria.Id] : new List<Guid>();
                            AddCompaniesInResult(companiesInResult, currentFilterCriteria, companies, noDataCompanies);
                            break;

                        case CriteriaTypeEnum.CriteriaType8:
                            var typeEight = new CriteriaTypeEight();
                            var criteriaTypeEight = JsonConvert.DeserializeAnonymousType(currentFilterCriteria.SelectedValue, typeEight);
                            criteriaTypeEight.CriteriaEntityField = currentFilterCriteria.CriteriaEntityField;
                            criteriaTypeEight.TemplateIndicatorNumber = currentFilterCriteria.Name.Split(' ')[0];

                            companies = ProcessCriteriaTypeEight(criteriaTypeEight, portfolioId, univs, _dataCompanies.ContainsKey(currentFilterCriteria.Id) ? _dataCompanies[currentFilterCriteria.Id] : new List<Guid>());
                            noDataCompanies = typeEightNoDataCompanies.ContainsKey(currentFilterCriteria.Id) ? typeEightNoDataCompanies[currentFilterCriteria.Id] : new List<Guid>();
                            AddCompaniesInResult(companiesInResult, currentFilterCriteria, companies, noDataCompanies);
                            break;

                        case CriteriaTypeEnum.CriteriaType9:
                            var typeNine = new CriteriaTypeNine();
                            var criteriaTypeNine = JsonConvert.DeserializeAnonymousType(currentFilterCriteria.SelectedValue, typeNine);
                            criteriaTypeNine.CriteriaEntityField = currentFilterCriteria.CriteriaEntityField;
                            criteriaTypeNine.TemplateIndicatorNumber = currentFilterCriteria.Name.Split(' ')[0];

                            companies = ProcessCriteriaTypeNine(criteriaTypeNine, portfolioId, univs);
                            noDataCompanies = typeNineNoDataCompanies.ContainsKey(currentFilterCriteria.Id) ? typeNineNoDataCompanies[currentFilterCriteria.Id] : new List<Guid>();
                            AddCompaniesInResult(companiesInResult, currentFilterCriteria, companies, noDataCompanies);
                            break;

                        case CriteriaTypeEnum.CriteriaType16:
                            var type16 = new CriteriaType16();
                            var criteriaType16 = JsonConvert.DeserializeAnonymousType(currentFilterCriteria.SelectedValue, type16);
                            criteriaType16.CriteriaEntityField = currentFilterCriteria.CriteriaEntityField;

                            var criteriaCompanies = _dataCompanies.ContainsKey(currentFilterCriteria.Id) ? _dataCompanies[currentFilterCriteria.Id] : new List<Guid>();
                            companies = ProcessCriteriaType16(criteriaType16, portfolioId, univs, criteriaCompanies);
                            noDataCompanies = type16NoDataCompanies.ContainsKey(currentFilterCriteria.Id) ? type16NoDataCompanies[currentFilterCriteria.Id] : new List<Guid>();
                            AddCompaniesInResult(companiesInResult, currentFilterCriteria, companies, noDataCompanies);
                            break;

                        case CriteriaTypeEnum.CriteriaType17:
                            var type17 = new CriteriaType17();
                            var criteriaType17 = JsonConvert.DeserializeAnonymousType(currentFilterCriteria.SelectedValue, type17);
                            criteriaType17.CriteriaEntityField = currentFilterCriteria.CriteriaEntityField;
                            criteriaType17.TemplateIndicatorNumber = currentFilterCriteria.Name.Split(' ')[0];

                            companies = ProcessCriteriaType17(criteriaType17, portfolioId, univs);
                            noDataCompanies = type17NoDataCompanies.ContainsKey(currentFilterCriteria.Id) ? type17NoDataCompanies[currentFilterCriteria.Id] : new List<Guid>();
                            AddCompaniesInResult(companiesInResult, currentFilterCriteria, companies, noDataCompanies);
                            break;
                    }
                }
                else
                {
                    AddCompaniesInResult(companiesInResult, currentFilterCriteria, new List<Guid>(), new List<Guid>());
                }
            });
        }

        private static void AddCompaniesInResult(List<CompaniesInResultInformation> companiesInResult,
            FilterCriteria currentFilterCriteria, List<Guid> companies, List<Guid> noDataCompanies)
        {
            lock (companiesInResult)
                companiesInResult.Add(new CompaniesInResultInformation
                {
                    CriteriaCompanies = companies,
                    CriteriaId = currentFilterCriteria.CriteriaLevel2Id,
                    CriteriaName = currentFilterCriteria.Name,
                    FilterCriteriaGroupId = currentFilterCriteria.FilterGroupId,
                    FilterCriteriaId = currentFilterCriteria.Id,
                    FilterCriteriaOrder = currentFilterCriteria.Order,
                    FilterSelectedValueInformation = currentFilterCriteria.SelectedValueInformation,
                    IsDataPoint = currentFilterCriteria.IsDataPoint,
                    CriteriaType = currentFilterCriteria.CriteriaType,
                    CriteriaEntityField = currentFilterCriteria.CriteriaEntityField,
                    NoDataCompanies = noDataCompanies,
                    ResearchProducts = currentFilterCriteria.ResearchProducts,
                    IsTemporary = currentFilterCriteria.IsTemporary
                });
        }

        private Dictionary<Guid, List<Guid>> NewProcessCriteria(List<FilterCriteria> currentFilterCriterias, CriteriaTypeEnum criteriaType, Filter filter, ScreeningReportData screeningReportData)
        {
            // Group by filter criteria id
            var filterCriteriaForCriteriaType = currentFilterCriterias.Where(p => p.CriteriaType == criteriaType);
            var criterias = filterCriteriaForCriteriaType.ToDictionary(k => k.Id, v => v);

            // prepare set fields string list
            var str = filterCriteriaForCriteriaType.Select(p => p.CriteriaEntityField.Replace(".", "")).ToList();
            var criteria = currentFilterCriterias.FirstOrDefault(p => p.CriteriaType == criteriaType);

            var companiesForCriterias = new Dictionary<Guid, List<Guid>>();
            if (criteria != null)
            {
                List<BsonDocument> indicatorsForCompanies;
                if (screeningReportData == null)
                {
                    // get universe for this criteria type
                    var _univs =
                        ClaimsPrincipal.Current.Claims.Any(c => c.Type == CustomClaimTypes.AccountId)
                            ? ClaimsPrincipal.Current.Claims.GetUniverseIds()
                            : _universeRepository.GetUniversesByAccountId(filter.AccountId).Select(p => p.Id).ToList();
                    List<Guid> companiesById = null;
                    if (filter != null && filter.PortfolioId != null)
                    {
                        companiesById = _portfolioReadHandler.GetCompanyIdsByPortfolio(filter.PortfolioId.Value).ToList();
                    }

                    // get indicator data
                    indicatorsForCompanies = _screeningDataRepository.ListCriteriaResults(str, _univs.ToList(), filter, companiesById);
                }
                else
                    indicatorsForCompanies = screeningReportData.CompanyData;

                Parallel.ForEach(criterias.Where(a=> !a.Value.IsTemporary), _criteria =>
                {
                    if (_criteria.Value.SelectedValue != null)
                    {
                        switch (criteriaType)
                        {
                            case CriteriaTypeEnum.CriteriaType4:
                                Parallel.ForEach(indicatorsForCompanies, indicatorCompany =>
                                {
                                    NewProcessTypeFourCriteria(indicatorCompany, _criteria, companiesForCriterias, _dataCompanies);
                                });
                                break;

                            case CriteriaTypeEnum.CriteriaType5:
                                var typeFive = new CriteriaTypeFive();
                                var criteriaTypeFive = JsonConvert.DeserializeAnonymousType(_criteria.Value.SelectedValue, typeFive);
                                Parallel.ForEach(indicatorsForCompanies, indicatorCompany =>
                                {
                                    NewProcessTypeFiveCriteria(indicatorCompany, _criteria, criteriaTypeFive, companiesForCriterias);
                                });
                                break;
                            case CriteriaTypeEnum.CriteriaType25:
                                var type25 = new CriteriaType25();
                                var criteriaType25 = JsonConvert.DeserializeAnonymousType(_criteria.Value.SelectedValue, type25);
                                Parallel.ForEach(indicatorsForCompanies, indicatorCompany =>
                                {
                                    NewProcessType25Criteria(indicatorCompany, _criteria, criteriaType25, companiesForCriterias);
                                });
                                break;

                            case CriteriaTypeEnum.CriteriaType7:
                                Parallel.ForEach(indicatorsForCompanies, indicatorCompany =>
                                {
                                    NewProcessTypeSevenCriteria(indicatorCompany, _criteria, companiesForCriterias, _dataCompanies);
                                });
                                break;

                            case CriteriaTypeEnum.CriteriaType9:
                                Parallel.ForEach(indicatorsForCompanies, indicatorCompany =>
                                {
                                    NewProcessTypeNineCriteria(indicatorCompany, _criteria, companiesForCriterias);
                                });
                                break;
                               
                            case CriteriaTypeEnum.CriteriaType16:
                            case CriteriaTypeEnum.CriteriaType17:
                                Parallel.ForEach(indicatorsForCompanies, indicatorCompany =>
                                {
                                    NewProcessCriteria(indicatorCompany, _criteria, companiesForCriterias);
                                });
                                break;

                            default:
                                Parallel.ForEach(indicatorsForCompanies, indicatorCompany =>
                                {
                                    NewProcessOtherCriteria(indicatorCompany, _criteria, companiesForCriterias, _dataCompanies);
                                });
                                break;
                        }
                    }
                });
            }

            return companiesForCriterias;
        }

        private void NewProcessTypeFiveCriteria(BsonDocument indicatorCompany,
            KeyValuePair<Guid, FilterCriteria> criteria,
            CriteriaTypeFive criteriaTypeFive, Dictionary<Guid, List<Guid>> companiesForCriterias)
        {
            var key = criteria.Value.CriteriaEntityField;
            // compute valid companies

            if (!indicatorCompany.Contains(key))
            {
                lock (_typeFiveNoDataCompanies)
                    if (_typeFiveNoDataCompanies.ContainsKey(criteria.Key))
                    {
                        _typeFiveNoDataCompanies[criteria.Key].Add(indicatorCompany["_id"].AsGuid);
                    }
                    else
                    {
                        _typeFiveNoDataCompanies.Add(criteria.Key, new List<Guid>());
                        _typeFiveNoDataCompanies[criteria.Key].Add(indicatorCompany["_id"].AsGuid);
                    }
            }

            if (indicatorCompany.Contains(key))
                if (indicatorCompany[key].IsString &&
                    (string.Equals(indicatorCompany[key].AsString, "No data") ||
                    string.Equals(indicatorCompany[key].AsString, "N/A")))
                {
                    lock (_typeFiveNoDataCompanies)
                        if (_typeFiveNoDataCompanies.ContainsKey(criteria.Key))
                        {
                            _typeFiveNoDataCompanies[criteria.Key].Add(indicatorCompany["_id"].AsGuid);
                        }
                        else
                        {
                            _typeFiveNoDataCompanies.Add(criteria.Key, new List<Guid>());
                            _typeFiveNoDataCompanies[criteria.Key].Add(indicatorCompany["_id"].AsGuid);
                        }
                }
                else
                {
                    if (criteriaTypeFive.Score != null && indicatorCompany[key].AsInt32 > criteriaTypeFive.Score.Value)
                    {
                        lock (companiesForCriterias)
                            if (companiesForCriterias.ContainsKey(criteria.Key))
                            {
                                companiesForCriterias[criteria.Key].Add(indicatorCompany["_id"].AsGuid);
                            }
                            else
                            {
                                companiesForCriterias.Add(criteria.Key, new List<Guid>());
                                companiesForCriterias[criteria.Key].Add(indicatorCompany["_id"].AsGuid);
                            }
                    }
                    // compute for 100 indicator score
                    else if (indicatorCompany[key].AsInt32 == 100)
                    {
                        lock (companiesForCriterias)
                            if (companiesForCriterias.ContainsKey(criteria.Key))
                            {
                                companiesForCriterias[criteria.Key].Add(indicatorCompany["_id"].AsGuid);
                            }
                            else
                                companiesForCriterias.Add(criteria.Key, new List<Guid>());
                    }
                }
        }

        private void NewProcessType25Criteria(BsonDocument indicatorCompany,
            KeyValuePair<Guid, FilterCriteria> criteria,
            CriteriaType25 criteriaType25, Dictionary<Guid, List<Guid>> companiesForCriterias)
        {
            var key = criteria.Value.CriteriaEntityField.Replace(".", "");
            // compute valid companies

            if (!indicatorCompany.Contains(key))
            {
                lock (_type25NoDataCompanies)
                    if (_type25NoDataCompanies.ContainsKey(criteria.Key))
                    {
                        _type25NoDataCompanies[criteria.Key].Add(indicatorCompany["_id"].AsGuid);
                    }
                    else
                    {
                        _type25NoDataCompanies.Add(criteria.Key, new List<Guid>());
                        _type25NoDataCompanies[criteria.Key].Add(indicatorCompany["_id"].AsGuid);
                    }
            }

            if (indicatorCompany.Contains(key))
                if (indicatorCompany[key].IsString &&
                    (string.Equals(indicatorCompany[key].AsString, "No data") ||
                    string.Equals(indicatorCompany[key].AsString, "N/A")))
                {
                    lock (_type25NoDataCompanies)
                        if (_type25NoDataCompanies.ContainsKey(criteria.Key))
                        {
                            _type25NoDataCompanies[criteria.Key].Add(indicatorCompany["_id"].AsGuid);
                        }
                        else
                        {
                            _type25NoDataCompanies.Add(criteria.Key, new List<Guid>());
                            _type25NoDataCompanies[criteria.Key].Add(indicatorCompany["_id"].AsGuid);
                        }
                }
                else
                {
                    if (criteriaType25.Score != null && indicatorCompany[key].AsInt32 < criteriaType25.Score.Value)
                    {
                        lock (companiesForCriterias)
                            if (companiesForCriterias.ContainsKey(criteria.Key))
                            {
                                companiesForCriterias[criteria.Key].Add(indicatorCompany["_id"].AsGuid);
                            }
                            else
                            {
                                companiesForCriterias.Add(criteria.Key, new List<Guid>());
                                companiesForCriterias[criteria.Key].Add(indicatorCompany["_id"].AsGuid);
                            }
                    }
                    // compute for 100 indicator score
                    else if (indicatorCompany[key].AsInt32 == 0)
                    {
                        lock (companiesForCriterias)
                            if (companiesForCriterias.ContainsKey(criteria.Key))
                            {
                                companiesForCriterias[criteria.Key].Add(indicatorCompany["_id"].AsGuid);
                            }
                            else
                                companiesForCriterias.Add(criteria.Key, new List<Guid>());
                    }
                }
        }

        private void NewProcessTypeFourCriteria(BsonDocument indicatorCompany,
            KeyValuePair<Guid, FilterCriteria> criteria,
            Dictionary<Guid, List<Guid>> notMeetsCriteriaCompanies,
            Dictionary<Guid, List<Guid>> meetsCriteriaCompanies)
        {
            var researchStates = indicatorCompany.GetResearchStates();

            // compute valid companies
            if (!indicatorCompany.Contains(criteria.Value.CriteriaEntityField))
            {
                if (researchStates.ProductInvolvement == ResearchState.ResearchAvailable
                    || researchStates.Solutions == ResearchState.ResearchAvailable)
                {
                    lock (meetsCriteriaCompanies)
                        if (meetsCriteriaCompanies.ContainsKey(criteria.Key))
                        {
                            meetsCriteriaCompanies[criteria.Key].Add(indicatorCompany["_id"].AsGuid);
                        }
                        else
                        {
                            meetsCriteriaCompanies.Add(criteria.Key, new List<Guid>());
                            meetsCriteriaCompanies[criteria.Key].Add(indicatorCompany["_id"].AsGuid);
                        }
                }
                else
                {
                    lock (notMeetsCriteriaCompanies)
                        if (notMeetsCriteriaCompanies.ContainsKey(criteria.Key))
                        {
                            notMeetsCriteriaCompanies[criteria.Key].Add(indicatorCompany["_id"].AsGuid);
                        }
                        else
                        {
                            notMeetsCriteriaCompanies.Add(criteria.Key, new List<Guid>());
                            notMeetsCriteriaCompanies[criteria.Key].Add(indicatorCompany["_id"].AsGuid);
                        }
                }
            }
            else
            {
                if (string.Equals(indicatorCompany[criteria.Value.CriteriaEntityField.Replace(".", "")].AsString, "No data"))
                {
                    lock (notMeetsCriteriaCompanies)
                        if (notMeetsCriteriaCompanies.ContainsKey(criteria.Key))
                        {
                            notMeetsCriteriaCompanies[criteria.Key].Add(indicatorCompany["_id"].AsGuid);
                        }
                        else
                        {
                            notMeetsCriteriaCompanies.Add(criteria.Key, new List<Guid>());
                            notMeetsCriteriaCompanies[criteria.Key].Add(indicatorCompany["_id"].AsGuid);
                        }
                }
                else
                {
                    lock (meetsCriteriaCompanies)
                        if (meetsCriteriaCompanies.ContainsKey(criteria.Key))
                        {
                            meetsCriteriaCompanies[criteria.Key].Add(indicatorCompany["_id"].AsGuid);
                        }
                        else
                        {
                            meetsCriteriaCompanies.Add(criteria.Key, new List<Guid>());
                            meetsCriteriaCompanies[criteria.Key].Add(indicatorCompany["_id"].AsGuid);
                        }
                }
            }
        }

        private void NewProcessTypeSevenCriteria(BsonDocument indicatorCompany,
            KeyValuePair<Guid, FilterCriteria> criteria,
            Dictionary<Guid, List<Guid>> companiesForCriterias,
            Dictionary<Guid, List<Guid>> dataCompaniesForCriterias)
        {
            // compute valid companies
            var key = criteria.Value.Name.Replace(".", "");
            if (indicatorCompany.Contains(key) && indicatorCompany[key].AsInt32 != 9999)
            {
                lock (dataCompaniesForCriterias)
                    if (dataCompaniesForCriterias.ContainsKey(criteria.Key))
                    {
                        dataCompaniesForCriterias[criteria.Key].Add(indicatorCompany["_id"].AsGuid);
                    }
                    else
                    {
                        dataCompaniesForCriterias.Add(criteria.Key, new List<Guid>());
                        dataCompaniesForCriterias[criteria.Key].Add(indicatorCompany["_id"].AsGuid);
                    }

                return;
            }

            lock (companiesForCriterias)
                if (companiesForCriterias.ContainsKey(criteria.Key))
                {
                    companiesForCriterias[criteria.Key].Add(indicatorCompany["_id"].AsGuid);
                }
                else
                {
                    companiesForCriterias.Add(criteria.Key, new List<Guid>());
                    companiesForCriterias[criteria.Key].Add(indicatorCompany["_id"].AsGuid);
                }
        }

        private static void NewProcessTypeNineCriteria(BsonDocument indicatorCompany,
            KeyValuePair<Guid, FilterCriteria> criteria,
            Dictionary<Guid, List<Guid>> companiesForCriterias)
        {
            if (indicatorCompany.HasControversyResearchAvailable())
                return;

            // compute valid companies
            if (!indicatorCompany.Contains(criteria.Value.CriteriaEntityField))
            {
                lock (companiesForCriterias)
                    if (companiesForCriterias.ContainsKey(criteria.Key))
                    {
                        companiesForCriterias[criteria.Key].Add(indicatorCompany["_id"].AsGuid);
                    }
                    else
                    {
                        companiesForCriterias.Add(criteria.Key, new List<Guid>());
                        companiesForCriterias[criteria.Key].Add(indicatorCompany["_id"].AsGuid);
                    }
            }
        }
        

        private static void NewProcessCriteria(BsonDocument indicatorCompany,
            KeyValuePair<Guid, FilterCriteria> criteria,
            Dictionary<Guid, List<Guid>> companiesForCriterias)
        {
            // compute valid companies
            if (!indicatorCompany.Contains(criteria.Value.CriteriaEntityField))
            {
                lock (companiesForCriterias)
                    if (companiesForCriterias.ContainsKey(criteria.Key))
                    {
                        companiesForCriterias[criteria.Key].Add(indicatorCompany["_id"].AsGuid);
                    }
                    else
                    {
                        companiesForCriterias.Add(criteria.Key, new List<Guid>());
                        companiesForCriterias[criteria.Key].Add(indicatorCompany["_id"].AsGuid);
                    }
            }
        }

        private static void NewProcessOtherCriteria(BsonDocument indicatorCompany,
            KeyValuePair<Guid, FilterCriteria> criteria,
            Dictionary<Guid, List<Guid>> companiesForCriterias,
            Dictionary<Guid, List<Guid>> dataCompaniesForCriterias)
        {
            // compute valid companies
            var key = criteria.Value.CriteriaEntityField.Replace(".", "");
            var keyIssue = "Issue - "+criteria.Value.CriteriaEntityField;
            if (!indicatorCompany.Contains(key) && indicatorCompany.Contains(keyIssue))
                key = keyIssue;

            if (indicatorCompany.Contains(key)  &&
                (!indicatorCompany[key].IsString ||
                (indicatorCompany[key].IsString &&
                indicatorCompany[key].AsString != "No data" &&
                indicatorCompany[key].AsString != "N/A" &&
                indicatorCompany[key].AsString != "")))
            {
                lock (dataCompaniesForCriterias)
                    if (dataCompaniesForCriterias.ContainsKey(criteria.Key))
                    {
                        dataCompaniesForCriterias[criteria.Key].Add(indicatorCompany["_id"].AsGuid);
                    }
                    else
                    {
                        dataCompaniesForCriterias.Add(criteria.Key, new List<Guid>());
                        dataCompaniesForCriterias[criteria.Key].Add(indicatorCompany["_id"].AsGuid);
                    }
                return;
            }

            lock (companiesForCriterias)
                if (companiesForCriterias.ContainsKey(criteria.Key))
                {
                    companiesForCriterias[criteria.Key].Add(indicatorCompany["_id"].AsGuid);
                }
                else
                {
                    companiesForCriterias.Add(criteria.Key, new List<Guid>());
                    companiesForCriterias[criteria.Key].Add(indicatorCompany["_id"].AsGuid);
                }
        }

        public Dictionary<Guid, BsonDocument> GetScreeningDataFields(Guid filterId, List<Guid> companies)
        {
            var helper = new CUSIPISINClaimsCheckerBL();
            helper.checkIsinCusipClaims(ClaimsPrincipal.Current);

            return _screeningDataRepository.GetScreeningDataDictionary(_companyScreeningRepository.GetFieldsForFilterCriteria(filterId), companies, helper.HasISINClaim, helper.HasCUSIPClaim);
        }

        public List<BsonDocument> GetResultOutput(ResultOutputFilter filter, int skip, int take, string[] orderBy = null, bool asc = true)
        {
            return _companyScreeningRepository.GetResultOutput(filter, skip, take, orderBy, asc);
        }

        public List<CriteriaResultOutput> GetCriteriaResultOutput(Guid filterId)
        {
            return _companyScreeningRepository.GetResultOutputByFilterId(filterId);
        }

        public List<BsonDocument> GetFullResultOutput(ResultOutputFilter filter)
        {
            return _companyScreeningRepository.GetFullResultOutput(filter);
        }

        public int CountResultOutput(ResultOutputFilter filter)
        {
            return _companyScreeningRepository.CountResultOutput(filter);
        }

        public void ClearResultOutput(Guid filterId)
        {
            try
            {
                CompanyScreeningSerilog.Log.ClearResultOutput(filterId);
                _companyScreeningRepository.DeleteResultOutput(filterId);
                CompanyScreeningSerilog.Log.ClearResultOutputSuccess(filterId);
            }
            catch (Exception ex)
            {
                CompanyScreeningSerilog.Log.ClearResultOutputError(filterId, ex);
                throw;
            }
        }

        #endregion

        #region utils

        private List<CriteriaChartData> ComputeCriteriaTypeOneData(CriteriaTypeOne criteriaTypeOne, List<int> universeIds)
        {
            try
            {
                CompanyScreeningSerilog.Log.LoadCriteriaTypeOneData(criteriaTypeOne.CriteriaEntityField);

                var numberOfClusters = int.Parse(ConfigurationManager.AppSettings["numberOfClusters"]);//todo  take this to ctor - Petre

                var rawData = _screeningDataRepository.LoadData(universeIds, criteriaTypeOne.CriteriaEntityField).ToList();

                if (rawData.Count == 0)
                    return new List<CriteriaChartData>();

                var zeroNumbers = 0;
                var logarithmValues = new List<double>();

                rawData.ForEach(raw =>
                {
                    if (raw.NearlyEq(0))
                    {
                        zeroNumbers++;
                        return;
                    }

                    var absoluteValue = Math.Abs(raw);
                    var logarithmValue = Math.Log(absoluteValue);
                    logarithmValues.Add(raw < 0 ? -logarithmValue : logarithmValue);
                });

                var clusterSize = (logarithmValues.Max() - logarithmValues.Min()) / numberOfClusters;

                CompanyScreeningSerilog.Log.ComputeClusters(criteriaTypeOne.CriteriaEntityField);
                List<Sustainalytics.Entities.Cluster> clusters = new List<Sustainalytics.Entities.Cluster>();
                clusters.Add(new Sustainalytics.Entities.Cluster() { Start = logarithmValues.Min(), End = logarithmValues.Min() + clusterSize, CompanyCount = 0 });
                for (int i = 1; i < numberOfClusters; i++)
                {
                    clusters.Add(new Sustainalytics.Entities.Cluster() { Start = clusters.Last().End, End = clusters.Last().End + clusterSize, CompanyCount = 0 });
                }

                foreach (double lnValue in logarithmValues)
                {
                    var clust = clusters.Where(c => lnValue > c.Start && lnValue <= c.End).SingleOrDefault();
                    if (clust == null)
                        clusters.First().CompanyCount++;
                    else
                        clust.CompanyCount++;
                }

                var zeroClust = clusters.Where(c => c.Start < 0 && c.End >= 0).SingleOrDefault();
                if (zeroClust != null)
                    zeroClust.CompanyCount += zeroNumbers;

                var data = new List<CriteriaChartData>();
                data.Add(new CriteriaChartData()
                {
                    XAxisValue = clusters.First().Start < 0 ?
                                 -(int)Math.Exp(Math.Abs(clusters.First().Start)) :
                                 (int)Math.Exp(Math.Abs(clusters.First().Start)),
                    YAxisValue = clusters.First().CompanyCount
                });

                for (int i = 1; i < clusters.Count - 1; i++)
                {
                    var currentCluster = clusters[i];
                    var minValue = currentCluster.Start < 0 ? -(int)Math.Exp(Math.Abs(currentCluster.Start)) : (int)Math.Exp(Math.Abs(currentCluster.Start));
                    var maxValue = currentCluster.End < 0 ? -(int)Math.Exp(Math.Abs(currentCluster.End)) : (int)Math.Exp(Math.Abs(currentCluster.End));

                    data.Add(new CriteriaChartData()
                    {
                        XAxisValue = (minValue + maxValue) / 2,
                        YAxisValue = currentCluster.CompanyCount
                    });
                }

                data.Add(new CriteriaChartData()
                {
                    XAxisValue = clusters.Last().End < 0 ?
                                 -(int)Math.Exp(Math.Abs(clusters.Last().End)) :
                                 (int)Math.Exp(Math.Abs(clusters.Last().End)),
                    YAxisValue = clusters.Last().CompanyCount
                });

                var trimmedData = data.Where(c => c.YAxisValue > 0).ToList();

                //Concates companies which have the same xaxisvalue.
                var summedData = new List<CriteriaChartData>();
                trimmedData.ForEach((td) =>
                {
                    var sd = summedData.Where(s => s.XAxisValue == td.XAxisValue).SingleOrDefault();
                    if (sd == null)
                        summedData.Add(td);
                    else
                        sd.YAxisValue += td.YAxisValue;
                });

                return summedData;
            }
            catch (Exception ex)
            {
                CompanyScreeningSerilog.Log.LoadCriteriaTypeOneDataError(criteriaTypeOne.CriteriaEntityField, ex);
                throw;
            }
        }

        private List<CriteriaChartData> ComputeCriteriaTypeSixData(CriteriaTypeSix criteriaTypeSix, List<int> universeIds)
        {
            try
            {
                CompanyScreeningSerilog.Log.LoadCriteriaTypeSixData(criteriaTypeSix.CriteriaEntityField);

                var rawData = _screeningDataRepository.LoadData(universeIds, criteriaTypeSix.CriteriaEntityField).ToList();
                var minData = criteriaTypeSix.MinValue;
                var maxData = criteriaTypeSix.MaxValue == 0 ? 100 : criteriaTypeSix.MaxValue;
                var dataList = new List<CriteriaChartData>();

                if (rawData == null || rawData.Count == 0)
                {
                    for (var i = minData; i <= maxData; i++)
                    {
                        dataList.Add(new CriteriaChartData { XAxisValue = i, YAxisValue = 0 });
                    }
                    return dataList;
                }

                if (!criteriaTypeSix.HasFullRangeChart)
                {
                    minData = (int) Math.Round(rawData.Min(), MidpointRounding.AwayFromZero);
                    maxData = (int) Math.Round(rawData.Max(), MidpointRounding.AwayFromZero);

                    if (minData != 0)
                        minData--;
                    if (maxData != 100)
                        maxData++;
                }

                for (var i = minData; i <= maxData; i++)
                {
                    dataList.Add(new CriteriaChartData { XAxisValue = i, YAxisValue = 0 });
                }

                if (criteriaTypeSix.CriteriaEntityField.EndsWith("Percentile", StringComparison.Ordinal))
                    rawData.ForEach((data) =>
                    {
                        var dl = dataList.Where(d => d.XAxisValue.NearlyEq(Math.Round(data*100, MidpointRounding.AwayFromZero))).SingleOrDefault();
                        if (dl != null)
                            dl.YAxisValue++;
                    });
                else
                    rawData.ForEach((data) =>
                    {
                        var dl = dataList.Where(d => d.XAxisValue.NearlyEq(Math.Round(data, MidpointRounding.AwayFromZero))).SingleOrDefault();
                        if (dl != null)
                            dl.YAxisValue++;
                    });
                return dataList;
            }
            catch (Exception ex)
            {
                CompanyScreeningSerilog.Log.LoadCriteriaTypeSixDataError(criteriaTypeSix.CriteriaEntityField, criteriaTypeSix.WeightMatrixId, ex);
                throw;
            }
        }

        private List<CriteriaChartData> ComputeCriteriaType16Data(CriteriaType16 criteriaType16, List<int> universeIds)
        {
            var rawData = _screeningDataRepository.LoadData(universeIds, criteriaType16.CriteriaEntityField).ToList();

            if (rawData == null || rawData.Count == 0)
                return new List<CriteriaChartData>();

            var minData = 0;
            var maxData = 100;

            if (!criteriaType16.HasFullRangeChart)
            {
                minData = (int)Math.Round(rawData.Min(), MidpointRounding.AwayFromZero);
                maxData = (int)Math.Round(rawData.Max(), MidpointRounding.AwayFromZero);

                if (minData != 0)
                    minData--;
                if (maxData != 100)
                    maxData++;
            }

            var dataList = new List<CriteriaChartData>();
            for (var i = minData; i <= maxData; i++)
            {
                dataList.Add(new CriteriaChartData { XAxisValue = i, YAxisValue = 0 });
            }

            rawData.ForEach((data) =>
            {
                var dl = dataList.SingleOrDefault(d => d.XAxisValue.NearlyEq(Math.Round(data, MidpointRounding.AwayFromZero)));
                if (dl != null)
                    dl.YAxisValue++;
            });

            return dataList;
        }

        #endregion

        #region TemplateCriteria

        public List<TemplateCriteria> ListTemplateCriteriaByLevel3Id(Guid? criteriaLevel3Id)
        {
            return _companyScreeningRepository.ListTemplateCriteriaByLevel3Id(criteriaLevel3Id);
        }

        public TemplateCriteria GetTemplateCriteriaById(Guid templateCriteriaId)
        {
            return _companyScreeningRepository.GetTemplateCriteriaById(templateCriteriaId);
        }

        public IndicatorTemplate GetIndicatorTemplate(string number)
        {
            return _clientDWRepository.GetIndicatorTemplate(number);
        }

        public GovernanceIndicatorTemplate GetGovernanceIndicatorTemplate(string number)
        {
            var indicatorTemplates = _governanceIndicatorTemplateRepository.ReadAllWhere(x => x.Number == number);

            //because some indicators were eliminated from Canvas we need to discuss whether or not the exception bellow should be thrown
            //if (indicatorTemplates == null)
            //    throw new Exception("The indicator template cannot be found");

            return indicatorTemplates != null ? indicatorTemplates.FirstOrDefault() : null;
        }

        //Return all template criteria as data points
        public List<TemplateCriteria> GetDataPointsByCriteriaLevel2IdAndChecks(Guid criteriaLevel2Id, List<TemplateCheck> checks)
        {
            return _companyScreeningRepository.GetDataPointsByCriteriaLevel2Id(criteriaLevel2Id);
        }

        public List<TemplateCriteria> ListTemplateCriteria()
        {
            try
            {
                var templateCriteria = _companyScreeningRepository.ListTemplateCriteria();
                return templateCriteria;
            }
            catch (Exception ex)
            {
                CompanyScreeningSerilog.Log.ListBaseCriteriaError(ex);
                throw;
            }
        }

        public List<AnswerCategory> GetAnswerCategoryForRiskRating(TemplateCriteria criteria)
        {
            List<AnswerCategory> listAnswer = new List<AnswerCategory>();
            List<RiskRatingCluster> listCluster = criteria.CriteriaEntityField.StartsWith("Issue - ")? _companyScreeningRepository.GetRiskRatingClusters_Mei() : _companyScreeningRepository.GetRiskRatingClusters_Overall();
            foreach (RiskRatingCluster clusterItem in listCluster)
            {
                listAnswer.Add(new AnswerCategory() { Score = (int)clusterItem.Assessment, Text = clusterItem.Assessment.ToString() });
            }
            return listAnswer;
        }

        #endregion
    }

    public class CompaniesInResultInformation
    {
        public Guid CriteriaId { get; set; }
        public List<Guid> CriteriaCompanies { get; set; }
        public string CriteriaName { get; set; }
        public Guid FilterCriteriaId { get; set; }
        public Guid? FilterCriteriaGroupId { get; set; }
        public string FilterSelectedValueInformation { get; set; }
        public int FilterCriteriaOrder { get; set; }
        public bool IsDataPoint { get; set; }
        public CriteriaTypeEnum CriteriaType { get; set; }

        [BsonRepresentation(BsonType.String)]
        public List<ResearchProduct> ResearchProducts { get; set; }
        public string CriteriaEntityField { get; set; }
        public List<Guid> NoDataCompanies { get; set; }
        public bool IsTemporary { get; set; }
    }
}
